/* Copyright 2008 Shawn C. Dodd

This file is part of TagFC.  TagFC is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.

TagFC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License along with TagFC.  If not, see <http://www.gnu.org/licenses/>. */

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.*;
import org.apache.commons.httpclient.params.HttpMethodParams;

class IngestSubscriptionsJob {
	def startDelay = 5000l		// race conditions are bad
	def timeout = 5000l		// how often
	def group = "TagFC"

	def quartzScheduler		// injected

	// Connect to inbound feed, download entries into database
	def execute() {
		Set newlySeen = new HashSet(21)
		int seenSize = 0

		// fetch the inbound feed details: URL, loop count, etc.
		def sub = Subscription.get(1)
		if( !sub ) log.error "Unable to get subscription"

		// hook up connection
		HttpClient client = new HttpClient()
		GetMethod method = new GetMethod(sub.url)
		method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));	// TODO  it'd be nice to extend the default handler to report problems to the error log
		if( sub.modification?.size() > 0 )
			method.setRequestHeader("If-Modified-Since", sub.modification)
		try {
			int statusCode = client.executeMethod(method)
			if( statusCode != HttpStatus.SC_OK ) {
				if( statusCode == HttpStatus.SC_NOT_MODIFIED ) {
					sub.loopCount = -1
//					log.error "Subscribed feed not modified"
				}
				else log.error "Request failed: ${method.getStatusLine()}"
			}
			InputStream body = method.getResponseBodyAsStream();
		
			// read the whole response
			def saveCount = 0
			if(body!=null) {		// HttpClient doc lied to me
                        def feedDoc = new XmlSlurper().parse(body)
			seenSize = sub.seen?.size()
			if( seenSize > 79 )
				log.error "Warning: Too many seen entries (${seenSize}) in subscription $sub.id."
			feedDoc.entry.each {
				// do a little processing
				def linkEnclosure=it.link.findAll{ it.@rel=="enclosure" }
				def contentHtml=it.content.findAll{ it.@type=="html" }
				def thumbMatch  = contentHtml =~ /img.src.*(http.*_m.jpg)/
				def widthMatch  = contentHtml =~ /width="(\d*)"/
				def heightMatch = contentHtml =~ /height="(\d*)"/
//				log.error "DEBUGGING: width='${widthMatch[0][1]}', height='${heightMatch[0][1]}'"

				// check to see if entry is in 'recently seen' list (a dupe)
				def newly = new DamnedEntry();
				newly.namespace = "flickr"
				newly.uri = it.id
				newlySeen.add(newly)
				for( i in sub.seen ) {
					if(newly.uri==i.uri && newly.namespace==i.namespace) {
//					log.error "Found a dupe!"
					return
					}
				}
				
				// insert entry into DB
				def simple = new SimpleEntry()
				simple.uri = it.id
				simple.namespace = "flickr"
				simple.fullImageURL = linkEnclosure.@href
				simple.thumbnailURL = thumbMatch[0][1]	// first one
				simple.thumbnailWidth = widthMatch[0][1]
				simple.thumbnailHeight = heightMatch[0][1]
				simple.tags = ""
				simple.markupFragment = ""	// TODO
				if(!simple.save()) log.error "Oops! Couldn't save SimpleEntry for ${it.id}"
				else {
					def piece = new Piece()
					piece.simpleEntry = simple
					piece.state = "unproc"
					if( piece.save() ) saveCount++
				}
			}
			
			// replace the old seen entries with the newly-seen ones
			def tempSet = new HashSet(sub.seen)	// blow up?
			tempSet.each { sub.removeFromSeen(it); it.delete() }
			newlySeen.each { 
				sub.addToSeen(namespace:it.namespace,uri:it.uri) 
			}

			// remember the modification date
			Header modDate = method.getResponseHeader("Last-Modified")
			sub.modification = modDate?.getValue() ?: ""
			}
			
			// should we kick off another poll?
			def shouldTrigger = false
			sub.loopCount--
			if( sub.loopCount > 0 ) {		// pardon my ugly state machine
				shouldTrigger = true
			} else if( sub.loopCount < 0 ) {
				sub.loopCount = 10	// TODO  don't hardcode this
			}
			if( !sub.save(flush:true) ) log.error "Oops, couldn't save subscription changes."
			log.error "Ingested $saveCount entries from feed $sub.id last modified '${sub.modification}' with ${seenSize} seen and $sub.loopCount remaining loops."
			if( shouldTrigger ) 
				quartzScheduler.triggerJobWithVolatileTrigger("IngestSubscriptionsJob", "TagFC")

		} catch( Exception e ) {
			log.error "Oops, couldn't process subscription."
			log.error e
			throw e
		} finally {
			method.releaseConnection();
		}
	}
}
class DamnedEntry {
	String namespace
	String uri
}
